In [ ]:
# Helper libraries
import datetime
import time
from packaging import version
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.ensemble import RandomForestClassifier
from collections import Counter
import numpy as np
import pandas as pd
from numpy.random import seed
# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow import keras
from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from tensorflow.keras.layers import Dropout, Flatten, Input, Dense
from tensorflow.keras.datasets import fashion_mnist
In [ ]:
print("This notebook requires TensorFlow 2.0 or above")
print("TensorFlow version: ", tf.__version__)
assert version.parse(tf.__version__).release[0] >=2
In [ ]:
print("Keras version: ", keras.__version__)
In [ ]:
%matplotlib inline
In [ ]:
def warn(*args, **kwargs):
pass
import warnings
warnings.warn = warn
In [ ]:
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
In [ ]:
print('train_images:\t{}'.format(train_images.shape))
print('train_labels:\t{}'.format(train_labels.shape))
print('test_images:\t\t{}'.format(test_images.shape))
print('test_labels:\t\t{}'.format(test_labels.shape))
In [ ]:
print("First ten labels training dataset:\n {}\n".format(train_labels[0:10]))
print("This output is the numeric label, need to convert to item description")
In [ ]:
# from collections import Counter
Counter(train_labels).most_common()
Out[ ]:
In [ ]:
Counter(test_labels).most_common()
Out[ ]:
In [ ]:
fig = plt.figure(figsize = (15, 9))
for i in range(4):
plt.subplot(2, 2, 1+i)
plt.title(train_labels[i])
plt.xticks([])
plt.yticks([])
plt.imshow(train_images[i].reshape(28,28), cmap='binary')
In [ ]:
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
In [ ]:
fig = plt.figure(figsize = (15, 9))
for i in range(50):
plt.subplot(5, 10, 1+i)
plt.title(class_names[train_labels[i]], fontsize=14)
plt.xticks([])
plt.yticks([])
plt.imshow(train_images[i].reshape(28,28), cmap='binary')
In [ ]:
np.set_printoptions(linewidth=np.inf)
print("{}".format(train_images[0]))
In [ ]:
train_images = train_images.reshape((60000, 28, 28, 1))
train_images_norm = train_images.astype('float32')/255
test_images = test_images.reshape((10000, 28, 28, 1))
test_images_norm = test_images.astype('float32')/255
In [ ]:
train_labels_encoded = to_categorical(train_labels)
test_labels_encoded = to_categorical(test_labels)
Deep Neural Net Experiments (w/ Adam Optimizer)¶
In [ ]:
def dnnModel(num_layers, hidden_units, hidden_activation, dropout, epochs, optimizer, name):
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Flatten())
for x in range(0,num_layers):
model.add(layers.Dense(units=hidden_units[x], activation=hidden_activation))
if dropout > 0:
model.add(layers.Dropout(dropout))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=epochs,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':name,
'Seed':5,
'Layers':num_layers,
'Hidden Nodes':hidden_units,
'Hidden Activation':hidden_activation,
'Dropout Rate':dropout,
'Epochs':epochs,
'Optimizer':optimizer,
'Time':total_time,
'Test Accuracy':accuracy
}
return (model,results)
def modelInsights(model):
preds = model.predict(test_images_norm)
plot = plt.figure(figsize = (12, 16))
start_index = 0
for i in range(25):
plot = plt.subplot(5, 5, i + 1)
plot = plt.grid(False)
plot = plt.xticks([])
plot = plt.yticks([])
pred = np.argmax(preds[start_index + i])
actual = np.argmax(test_labels_encoded[start_index + i])
col = 'g'
if pred != actual:
col = 'r'
plot = plt.xlabel('i={}\nPredicted={}\nActual={}'.format(start_index + i, class_names[pred], class_names[actual]), color = col)
plot = plt.imshow(test_images[start_index + i].reshape(28,28), cmap='binary')
print('First 25 Predictions')
print('--------------------')
plt.show()
print()
# Get the predicted classes (Test):
pred_test = np.argmax(model.predict(test_images_norm), axis=-1)
conf_mx = tf.math.confusion_matrix(test_labels, pred_test)
print('Confusion Matrix (Testing Set)')
print('------------------------------')
fig, ax = plt.subplots(figsize=(15,15))
ax = sns.heatmap(conf_mx, annot=True, fmt='g')
ax.set_xlabel("Predicted Classes")
ax.set_ylabel("Actual Classes")
ax.set_xticklabels(class_names)
ax.set_yticklabels(class_names, va='center')
plt.show()
print()
for class_index in range(0,10):
i = 0
fig = plt.figure(figsize=(12,6))
fig.suptitle('First 5 {} Images'.format(class_names[class_index]))
for image in np.where(test_labels==class_index)[0][0:10]:
ax = fig.add_subplot(2,5,i+1)
ax.set_xticks([])
ax.set_yticks([])
pred = np.argmax(preds[image])
actual = np.argmax(test_labels_encoded[image])
col = 'g'
if pred != actual:
col = 'r'
ax.set_xlabel('i={}\nPredicted={}\nActual={}'.format(image, class_names[pred], class_names[actual]), color = col)
ax.imshow(test_images[image].reshape(28,28), cmap='binary')
i = i+1
In [ ]:
dnn_results = pd.DataFrame(columns=['Model','Seed','Layers','Hidden Nodes','Hidden Activation','Dropout Rate','Epochs','Optimizer','Time','Test Accuracy'])
In [ ]:
node_options = [10, 20, 40, 80, 160, 320]
for dropout_option in [0,0.2]:
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp1_'+str(nodes)+'_Model'
else:
folder_name = 'Exp1_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=2,
hidden_units=[nodes,nodes],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp2_'+str(nodes)+'_Model'
else:
folder_name = 'Exp2_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=3,
hidden_units=[nodes,nodes,nodes],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp2b_'+str(nodes)+'_Model'
else:
folder_name = 'Exp2b_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=3,
hidden_units=[nodes,nodes/2,nodes/4],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
for nodes in node_options:
if dropout_option == 0:
folder_name = 'Exp2c_'+str(nodes)+'_Model'
else:
folder_name = 'Exp2c_'+str(nodes)+'_Model_Dropout'
exp_model = dnnModel(
num_layers=3,
hidden_units=[nodes,nodes,nodes/2],
hidden_activation='relu',
epochs=15,
optimizer='adam',
dropout=dropout_option,
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
dnn_results = dnn_results.append(exp_model[1],ignore_index=True)
dnn_results.to_csv('DNN_Results.csv')
dnn_results
Out[ ]:
In [ ]:
dnn_results.to_csv('/content/drive/My Drive/Colab Notebooks/DNN_Results.csv')
Experiment 1 (DNN w/ 2 Layers of Equal Size)¶
- No Dropout Layer
- Dropout Layer (20%)
In [ ]:
exp1_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp1_320_Model')
modelInsights(exp1_best)
In [ ]:
exp1_dropout_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp1_320_Model_Dropout')
modelInsights(exp1_dropout_best)
In [ ]:
dnn_results['Experiment'] = dnn_results['Model'].apply(lambda x: x.split('_')[0])
dnn_results['N'] = dnn_results['Hidden Nodes'].apply(lambda x: x[0])
In [ ]:
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp1']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
Experiment 2 (DNN w/ 3 Layers of Equal Size)¶
In [ ]:
exp2_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2_320_Model')
modelInsights(exp2_best)
In [ ]:
exp2_dropout_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2_320_Model_Dropout')
modelInsights(exp2_dropout_best)
In [ ]:
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp2']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
Experiment 2B (DNN w/ 3 Layers with Layer Sizes N -> N/2 -> N/4)¶
In [ ]:
exp2b_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2b_320_Model')
modelInsights(exp2b_best)
In [ ]:
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp2b']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
Experiment 2C (DNN w/ 3 Layers with Layer Sizes N -> N -> N/2)¶
In [ ]:
exp2c_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp2c_80_Model')
modelInsights(exp2c_best)
In [ ]:
plot_df = dnn_results[dnn_results['Experiment'] == 'Exp2c']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
Convoluted Neural Net Experiments¶
In [ ]:
cnn_results = pd.DataFrame(columns=['Model','Seed','Layers','Filters','Hidden Activation','Dropout Rate','Epochs','Optimizer','Time','Test Accuracy'])
In [ ]:
def cnnModel(num_layers, num_filters, hidden_activation, dropout, epochs, optimizer, name):
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
for x in range(0,num_layers):
model.add(layers.Conv2D(filters=num_filters[x], kernel_size=(3,3), strides=(1,1), activation=hidden_activation, input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation=hidden_activation))
if dropout > 0:
model.add(layers.Dropout(dropout))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=epochs,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {'Model':name,
'Seed':5,
'Layers':num_layers,
'Filters':num_filters,
'Hidden Activation':hidden_activation,
'Dropout Rate':dropout,
'Epochs':epochs,
'Optimizer':optimizer,
'Time':total_time,
'Test Accuracy':accuracy
}
print(total_time)
return (model,results)
In [ ]:
filter_options = [1, 2, 4, 8, 16, 32]
for dropout_option in [0,0.2]:
# Experiment 3
for filters in filter_options:
print('Running model with {} -> {} filters'.format(filters,filters*2))
if dropout_option == 0:
folder_name = 'Exp3_'+str(filters)+'_Model'
else:
folder_name = 'Exp3_'+str(filters)+'_Model_Dropout'
exp_model = cnnModel(
num_layers=2,
num_filters=[filters,(filters*2)],
hidden_activation='relu',
dropout=dropout_option,
epochs=15,
optimizer='adam',
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
cnn_results = cnn_results.append(exp_model[1],ignore_index=True)
cnn_results.to_csv('/content/drive/My Drive/Colab Notebooks/CNN_Results.csv')
# Experiment 4
for filters in filter_options:
print('Running model with {} -> {} -> {} filters'.format(filters,filters*2,filters*2))
if dropout_option == 0:
folder_name = 'Exp4_'+str(filters)+'_Model'
else:
folder_name = 'Exp4_'+str(filters)+'_Model_Dropout'
exp_model = cnnModel(
num_layers=3,
num_filters=[filters,(filters*2),(filters*4)],
hidden_activation='relu',
dropout=dropout_option,
epochs=15,
optimizer='adam',
name=folder_name
)
exp_model[0].save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
cnn_results = cnn_results.append(exp_model[1],ignore_index=True)
cnn_results.to_csv('/content/drive/My Drive/Colab Notebooks/CNN_Results.csv')
cnn_results
Out[ ]:
In [ ]:
Experiment 3 (CNN with 2 Convolution/Max Pooling Layers)¶
In [ ]:
exp3_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp3_32_Model')
modelInsights(exp3_best)
In [ ]:
cnn_results['Experiment'] = cnn_results['Model'].apply(lambda x: x.split('_')[0])
cnn_results['N'] = cnn_results['Filters'].apply(lambda x: x[0])
plot_df = cnn_results[cnn_results['Experiment'] == 'Exp3']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
In [ ]:
# Repeat Exp3 stopping at 6 Epochs
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=6,
validation_split=(1/60),
batch_size=100,
verbose=True)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/Exp3_32_Model_6Epochs')
Experiment 4 (CNN with 3 Convolution/Max Pooling Layers)¶
In [ ]:
exp4_best = keras.models.load_model('/content/drive/My Drive/Colab Notebooks/Exp4_32_Model')
modelInsights(exp4_best)
In [ ]:
plot_df = cnn_results[cnn_results['Experiment'] == 'Exp4']
fig, ax = plt.subplots(figsize=(10,10))
ax = sns.lineplot(x='N',y='Test Accuracy',data=plot_df,hue='Dropout Rate')
In [ ]:
exp5f
Experiment 3 Analysis¶
- Shirts vs. T-Shirts
- Visualizing Filters
In [ ]:
modelInsights(exp3_best)
In [ ]:
def plot_digits(instances, pos, images_per_row=5, **options):
size = 28
images_per_row = min(len(instances), images_per_row)
images = [instance.reshape(size,size) for instance in instances]
n_rows = (len(instances) - 1) // images_per_row + 1
row_images = []
n_empty = n_rows * images_per_row - len(instances)
images.append(np.zeros((size, size * n_empty)))
for row in range(n_rows):
rimages = images[row * images_per_row : (row + 1) * images_per_row]
row_images.append(np.concatenate(rimages, axis=1))
image = np.concatenate(row_images, axis=0)
pos.imshow(image, cmap = 'binary', **options)
pos.axis("off")
In [ ]:
pred_classes = np.argmax(exp3_best.predict(test_images_norm), axis=-1)
cl_a, cl_b = 0, 6
X_aa = test_images_norm[(test_labels == cl_a) & (pred_classes == cl_a)]
X_ab = test_images_norm[(test_labels == cl_a) & (pred_classes == cl_b)]
X_ba = test_images_norm[(test_labels == cl_b) & (pred_classes == cl_a)]
X_bb = test_images_norm[(test_labels == cl_b) & (pred_classes == cl_b)]
plt.figure(figsize=(28,28))
p1 = plt.subplot(221)
p2 = plt.subplot(222)
p3 = plt.subplot(223)
p4 = plt.subplot(224)
plot_digits(X_aa[:25], p1, images_per_row=5);
plot_digits(X_ab[:25], p2, images_per_row=5);
plot_digits(X_ba[:25], p3, images_per_row=5);
plot_digits(X_bb[:25], p4, images_per_row=5);
p1.set_title(f"{class_names[cl_a]}'s classified as {class_names[cl_a]}'s")
p2.set_title(f"{class_names[cl_a]}'s classified as {class_names[cl_b]}'s")
p3.set_title(f"{class_names[cl_b]}'s classified as {class_names[cl_a]}'s")
p4.set_title(f"{class_names[cl_b]}'s classified as {class_names[cl_b]}'s")
plt.show()
In [ ]:
# Extracts the outputs of the all the layers
layer_outputs = [layer.output for layer in exp3_best.layers]
# Creates a model that will return these outputs, given the model input:
activation_model = models.Model(inputs=exp3_best.input, outputs=layer_outputs)
layer_names = []
for layer in exp3_best.layers:
layer_names.append(layer.name)
layer_names
Out[ ]:
In [ ]:
from tensorflow.keras.preprocessing import image
(_,_), (test_images, test_labels) = fashion_mnist.load_data()
img = test_images[19]
img_tensor = image.img_to_array(img)
img_tensor = np.expand_dims(img_tensor, axis=0)
# https://www.tensorflow.org/tutorials/keras/classification
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
plt.imshow(img, cmap="binary")
plt.axis('off')
plt.title(class_names[test_labels[40]], fontsize=12)
plt.show()
In [ ]:
activations = activation_model.predict(img_tensor)
len(activations)
Out[ ]:
In [ ]:
# These are the names of the layers, so can have them as part of our plot
layer_names = []
for layer in exp3_best.layers:
layer_names.append(layer.name)
images_per_row = 16
# Now let's display our feature maps
for layer_name, layer_activation in zip(layer_names, activations):
if layer_name[0:7] == 'flatten':
break
# This is the number of features in the feature map
n_features = layer_activation.shape[-1]
# The feature map has shape (1, size, size, n_features)
size = layer_activation.shape[1]
# We will tile the activation channels in this matrix
n_cols = n_features // images_per_row
display_grid = np.zeros((size * n_cols, images_per_row * size))
# We'll tile each filter into this big horizontal grid
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,
:, :,
col * images_per_row + row]
# Post-process the feature to make it visually palatable
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255).astype('uint8')
display_grid[col * size : (col + 1) * size,
row * size : (row + 1) * size] = channel_image
# Display the grid
scale = 1. / size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
plt.grid(False)
plt.imshow(display_grid, aspect='auto', cmap='binary')
plt.show()
In [ ]:
keras.utils.plot_model(exp3_best, "Experiment3_Model.png", show_shapes=True)
Out[ ]:
Additional Experiments¶
Adjust Hidden Layer Size¶
In [ ]:
exp_denseSize_results = pd.DataFrame(columns=['Model','Dense Layer Size','Time','Test Accuracy'])
In [ ]:
for layer_size in [10, 20, 40, 80, 160, 320, 640]:
seed(5)
tf.random.set_seed(5)
folder_name = 'exp_denseSize_' + str(layer_size)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=layer_size, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':folder_name,
'Dense Layer Size':layer_size,
'Time':total_time,
'Test Accuracy':accuracy
}
exp_denseSize_results = exp_denseSize_results.append(results, ignore_index=True)
exp_denseSize_results.to_csv('/content/drive/My Drive/Colab Notebooks/denseSize_results.csv')
model.save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
exp_denseSize_results
Out[ ]:
Add Dense Layer¶
In [ ]:
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/addDenseLayer')
Adjust Layer Size w/ 2 Dense Layers¶
In [ ]:
exp_denseSize2_results = pd.DataFrame(columns=['Model','Dense Layer Size','Time','Test Accuracy'])
In [ ]:
for layer_size in [10, 20, 40, 80, 160, 320, 640]:
seed(5)
tf.random.set_seed(5)
folder_name = 'exp_denseSize2_' + str(layer_size)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=layer_size, activation='relu'))
model.add(layers.Dense(units=layer_size, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':folder_name,
'Dense Layer Size':layer_size,
'Time':total_time,
'Test Accuracy':accuracy
}
exp_denseSize2_results = exp_denseSize2_results.append(results, ignore_index=True)
exp_denseSize2_results.to_csv('/content/drive/My Drive/Colab Notebooks/denseSize2_results.csv')
model.save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
exp_denseSize2_results
Out[ ]:
In [ ]:
Increase Conv2D Strides to (2,2)¶
In [ ]:
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(2,2), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(2,2), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/increaseConvStride')
In [ ]:
seed(5)
tf.random.set_seed(5)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=3))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=3))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=100,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
print('Total Time:',total_time)
print('Test Accuracy:',accuracy)
model.save('/content/drive/My Drive/Colab Notebooks/increasePoolStride')
Adjust Batch Sizes¶
In [ ]:
exp_batchSize_results = pd.DataFrame(columns=['Model','Dense Layer Size','Time','Test Accuracy'])
for batch in [500, 1000, 5000, 10000, 50000]:
seed(5)
tf.random.set_seed(5)
folder_name = 'exp_batchSize_' + str(batch)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(2,2), strides=(1,1), activation='relu', input_shape=(28,28,1)))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu'))
model.add(layers.MaxPooling2D((2,2), strides=2))
model.add(layers.Flatten())
model.add(layers.Dense(units=1024, activation='relu'))
model.add(layers.Dense(units=10, activation='softmax'))
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
start_time = time.time()
history = model.fit(
x=train_images_norm,
y=train_labels_encoded,
epochs=15,
validation_split=(1/60),
batch_size=batch,
verbose=False)
losses = history.history['loss']
accs = history.history['accuracy']
val_losses = history.history['val_loss']
val_accs = history.history['val_accuracy']
epochs = len(losses)
plt.figure(figsize=(12, 4))
for i, metrics in enumerate(zip([losses, accs], [val_losses, val_accs], ['Loss', 'Accuracy'])):
plt.subplot(1, 2, i + 1)
plt.plot(range(epochs), metrics[0], label='Training {}'.format(metrics[2]))
plt.plot(range(epochs), metrics[1], label='Validation {}'.format(metrics[2]))
plt.legend()
plt.show()
total_time = time.time() - start_time
loss, accuracy = model.evaluate(test_images_norm, test_labels_encoded)
results = {
'Model':folder_name,
'Batch Size':batch,
'Time':total_time,
'Test Accuracy':accuracy
}
exp_batchSize_results = exp_batchSize_results.append(results, ignore_index=True)
exp_batchSize_results.to_csv('/content/drive/My Drive/Colab Notebooks/batchSize_results.csv')
model.save('/content/drive/My Drive/Colab Notebooks/'+folder_name)
exp_batchSize_results
In [ ]: